# encoding='utf-8'
"""
@Author: 张晟烨
@Date: 2022/08/03
@Email: zhangsy@zylliondata.com
@Description: simhash核心程序，包括hash数据和数据去重两个功能，如想看懂建议先阅读
              http://static.googleusercontent.com/media/research.google.com/en//pubs/archive/33026.pdf
@Version: Python3.8
@Modified By:

"""

from __future__ import division, unicode_literals
import collections
import hashlib
import logging
import numbers
import re
import sys
from itertools import groupby
import numpy as np

# 规范数据类型
try:
    from collections.abc import Iterable
except ImportError:
    from collections import Iterable

if sys.version_info[0] >= 3:
    basestring = str
    unicode = str
    long = int


# 整型到二进制
def int_to_bytes(n, length):
    """
    # 整型到二进制
    :param n:
    :param length:
    :return:
    """

    return n.to_bytes(length, 'big')


# 二进制到整型
def bytes_to_int(b):
    """
    二进制到整型
    :param b:
    :return:
    """
    return int.from_bytes(b, 'big')


#  md5数据加密,底层hash
def _hashfunc(x):
    """
    md5数据加密,底层hash
    :param x:
    :return:
    """
    return hashlib.md5(x).digest()


#  生成simhash值主程序
class Simhash(object):
    large_weight_cutoff = 50
    batch_size = 200

    def __init__(
            self, value, f=64, reg=r'[\w\u4e00-\u9fcc]+', hashfunc=_hashfunc, log=None
    ):
        """
        `f` hash值长度，要求为8的倍数.

        `reg` 处理后数据的正则表达式

        `hashfunc` 请确保输入字符串为可以被'utf-8'解析的字符串.
        """
        if f % 8:
            raise ValueError('f must be a multiple of 8')

        self.f = f
        self.f_bytes = f // 8
        self.reg = reg
        self.value = None
        self.hashfunc = hashfunc
        self.hashfunc_returns_int = isinstance(hashfunc(b"test"), numbers.Integral)

        if log is None:
            self.log = logging.getLogger("simhash")
        else:
            self.log = log

        if isinstance(value, Simhash):
            self.value = value.value
        elif isinstance(value, str):
            self.build_by_text(value)
        elif isinstance(value, dict):
            self.build_by_features(value)
        elif isinstance(value, list):
            self.build_by_features(value)
        elif isinstance(value, int):
            self.value = value
        else:
            raise Exception('Bad parameter with type {}'.format(type(value)))

    def __eq__(self, other):
        """
        Compare two simhashes by their value.

        :param Simhash other: The Simhash object to compare to
        """
        return self.value == other.value

    @staticmethod
    def _slide(content, width=4):
        """
        n-gram
        :param content: 字符串
        :param width:  ngram里面的n
        :return:
        """
        return [content[i:i + width] for i in range(max(len(content) - width + 1, 1))]

    def _tokenize(self, content):
        """
        分词
        :param content:
        :return:
        """
        content = content.lower()
        content = ''.join(re.findall(self.reg, content))
        ans = self._slide(content)
        return ans

    def build_by_text(self, content):
        """
        对用ngram处理过的文本，计算其hash值
        :param content: 文本
        :return: hash值
        """
        features = self._tokenize(content)
        features = {k: sum(1 for _ in g) for k, g in groupby(sorted(features))}
        return self.build_by_features(features)

    def build_by_features(self, features):
        """
        计算hash
        :param features: 字典、元组、list、字符串，我都支持

        :return: hash值
        """

        sums = []
        batch = []
        count = 0
        w = 1
        truncate_mask = 2 ** self.f - 1
        if isinstance(features, dict):
            features = features.items()

        for f in features:
            skip_batch = False
            if not isinstance(f, basestring):
                f, w = f
                skip_batch = w > self.large_weight_cutoff or not isinstance(w, int)

            count += w
            if self.hashfunc_returns_int:
                h = int_to_bytes(self.hashfunc(f.encode('utf-8')) & truncate_mask, self.f_bytes)
            else:
                h = self.hashfunc(f.encode('utf-8'))[-self.f_bytes:]

            if skip_batch:
                sums.append(self._bitarray_from_bytes(h) * w)
            else:
                batch.append(h * w)
                if len(batch) >= self.batch_size:
                    sums.append(self._sum_hashes(batch))
                    batch = []

            if len(sums) >= self.batch_size:
                sums = [np.sum(sums, 0)]

        if batch:
            sums.append(self._sum_hashes(batch))

        combined_sums = np.sum(sums, 0)
        self.value = bytes_to_int(np.packbits(combined_sums > count / 2).tobytes())

    def _sum_hashes(self, digests):
        """
        生成hash
        :param digests:
        :return:
        """
        bitarray = self._bitarray_from_bytes(b''.join(digests))
        rows = np.reshape(bitarray, (-1, self.f))
        return np.sum(rows, 0)

    @staticmethod
    def _bitarray_from_bytes(b):
        """

        :param b:
        :return:
        """
        return np.unpackbits(np.frombuffer(b, dtype='>B'))

    def distance(self, another):
        """
        :param another: 另外一个simhash值，类型为simhash
        :return: hamming distance
        """
        assert self.f == another.f
        x = (self.value ^ another.value) & ((1 << self.f) - 1)
        ans = 0
        while x:
            ans += 1
            x &= x - 1
        return ans


class Index(object, ):

    def __init__(self, input_data, f=64, k=3, log=None):
        """
        传入一个类似{1：xxxx,2:xxxx,3:xxxx}的长字典，其中xxxx要求为上一个类加工出的simhash值
        :param input_data: 输入的数据集合
        :param f: simhash位数
        :param k: 最大hamming_distacne
        :param log: 日志
        """

        self.k = k
        self.f = f
        count = len(input_data)

        if log is None:
            self.log = logging.getLogger("simhash")
        else:
            self.log = log

        self.log.info('Initializing %s data.', count)

        self.bucket = collections.defaultdict(set)

        for i, q in enumerate(input_data):
            if i % 10000 == 0 or i == count - 1:
                self.log.info('%s/%s', i + 1, count)

            self.add(*q)

    def get_near_dups(self, simhash):
        """
        建立index集合后，求单个simhash值在这个集合中的近似值
        :param simhash: 传入的单个simhash
        :return: 集合中重复数据的id
        """
        assert simhash.f == self.f

        ans = set()

        for key in self.get_keys(simhash):
            dups = self.bucket[key]
            self.log.debug('key:%s', key)

            for dup in dups:
                sim2, obj_id = dup.split(',', 1)
                sim2 = Simhash(long(sim2, 16), self.f)

                d = simhash.distance(sim2)
                if d <= self.k:
                    ans.add(obj_id)
        return list(ans)

    def add(self, obj_id, simhash):
        """
        在index集合中添加某个元素
        `obj_id` ：比如1，2，3，4，5，6（字符串格式）
        `simhash` is an instance of Simhash
        """
        assert simhash.f == self.f

        for key in self.get_keys(simhash):
            v = '%x,%s' % (simhash.value, obj_id)
            self.bucket[key].add(v)

    def delete(self, obj_id, simhash):
        """
        在index集合中删除某个元素
        `obj_id` ：比如1，2，3，4，5，6（字符串格式）
        `simhash` is an instance of Simhash
        """
        assert simhash.f == self.f

        for key in self.get_keys(simhash):
            v = '%x,%s' % (simhash.value, obj_id)
            if v in self.bucket[key]:
                self.bucket[key].remove(v)

    @property
    def offsets(self):
        """
        根据k值，划分bucket
        """
        return [self.f // (self.k + 1) * i for i in range(self.k + 1)]

    def get_keys(self, simhash):
        """
        :param simhash:传入的simhash值
        :return: 被分成四份的simhash值
        """
        for i, offset in enumerate(self.offsets):
            if i == (len(self.offsets) - 1):
                m = 2 ** (self.f - offset) - 1
            else:
                m = 2 ** (self.offsets[i + 1] - offset) - 1
            c = simhash.value >> offset & m
            yield '%x:%x' % (c, i)

    def bucket_size(self):
        """
        分箱
        :return:
        """
        return len(self.bucket)


if __name__ == '__main__':
    def get_features(s):
        """
        取得特征
        :param s:
        :return:
        """
        width = 3
        s = s.lower()
        s = re.sub(r'[^\w]+', '', s)
        return [s[i:i + width] for i in range(max(len(s) - width + 1, 1))]


    data = {
        1: '富临精工(300432.SZ)12月13日在投资者互动平台表示,公司磷酸铁锂正极材料主要应用于新能源汽车动力电池,也可以用于储能领域。公司技术创新、产品升级、产能布局与客户技术创新升级趋势及市场需求紧密结合,公司将持续推进技术迭代和产品升级。',
        2: '【太平洋汽车 行业频道】2020年,本田与宁德时代就新能源汽车动力电池签署全方位战略合作协议,双方合作内容广泛覆盖动力电池的共同开发、稳定供给、回收及再利用等领域。2022年11月,主要从事本田中国e:N品牌纯电动车动力电池集中采购业务的众锐...',
        3: '新能源汽车订单暴增 新能源汽车消费者排队提车 _ 东方财富网'
    }

    print(get_features(
        '富临精工(300432.SZ)12月13日在投资者互动平台表示,公司磷酸铁锂正极材料主要应用于新能源汽车动力电池,也可以用于储能领域。公司技术创新、产品升级、产能布局与客户'))
    inputs = [(str(k), Simhash(get_features(v))) for k, v in data.items()]
    index = Index(inputs, k=3)
    print(index.bucket)





    text1 = 'spongebob square_pants，spongebob square_pants，spongebob square_pants，spongebob square_pants22'
    #
    s1 = Simhash(get_features(text1))
    print(s1.value)
    s2 = Simhash(get_features('中国挖掘机网'))
    s3 = Simhash(get_features('中国挖掘机网'))
    #
    # print(index.get_near_dups(s1))
    #
    index.add('4', s1)
    index.add('5', s2)
    index.add('6', s3)
    # print(index.bucket)
    # print(len(index.bucket))
    print(index.get_near_dups(s3))


